const path = require("path");
const { createWriteStream } = require("fs");
const { promisify } = require("util");
const pipeline = promisify(require("stream").pipeline);
const crypto = require("crypto");

// 下载图片并按指定格式命名
async function downloadImage(url, newsId) {
  if (!url) {
    console.log("无图片链接");
    return null;
  }

  try {
    const response = await axios({ url, responseType: "stream" });
    const timestamp = Date.now(); // 时间戳
    const hash = crypto.createHash("md5").update(url).digest("hex"); // 哈希值
    const ext = path.extname(new URL(url).pathname) || ".jpg"; // 获取扩展名

    const filename = `${newsId}_${timestamp}_${hash}${ext}`;
    const filepath = path.join(__dirname, "images", filename);

    // 创建 images 文件夹（如果不存在）
    const dir = path.dirname(filepath);
    if (!fs.existsSync(dir)) {
      fs.mkdirSync(dir, { recursive: true });
    }

    await pipeline(response.data, createWriteStream(filepath));
    console.log(`图片已保存为: ${filename}`);
    return `/images/${filename}`; // 返回相对路径
  } catch (error) {
    console.error("下载图片失败:", error.message);
    return null;
  }
}

function getAbsoluteImageUrl(baseUrl, relativeUrl) {
  if (!relativeUrl) return null;
  try {
    return new URL(relativeUrl, baseUrl).href;
  } catch (e) {
    console.error("无法解析图片链接:", relativeUrl);
    return null;
  }
}

const axios = require("axios");
const cheerio = require("cheerio");
const mysql = require("mysql2");
const fs = require("fs");
const moment = require("moment");

// 创建 MySQL 连接
const connection = mysql.createConnection({
  host: "localhost", // 数据库地址
  user: "root", // 数据库用户名
  password: "123456", // 数据库密码
  database: "news_db", // 数据库名称
});

// 连接到数据库
connection.connect((err) => {
  if (err) {
    return console.error("连接失败:", err.stack);
  }
  console.log("连接成功.");
});

// 爬取新浪新闻首页
async function scrapeNews() {
  let connection; // 将 connection 放到函数作用域内管理

  try {
    const url = "https://news.sina.com.cn";
    const response = await axios.get(url);
    const html = response.data;
    const $ = cheerio.load(html);

    const newsItems = [];
    $("div#syncad_1 a").each((index, element) => {
      const title = $(element).text().trim();
      const link = $(element).attr("href");
      if (title && link) {
        newsItems.push({ title, link });
      }
    });

    if (newsItems.length === 0) {
      console.error("爬取失败！");
      return;
    }

    // 连接数据库
    connection = mysql.createConnection({
      host: "localhost",
      user: "root",
      password: "123456",
      database: "news_db"
    });

    for (const item of newsItems) {
      const detailResponse = await axios.get(item.link);
      const detailHtml = detailResponse.data;
      const detail$ = cheerio.load(detailHtml);

      const content = detail$("div.article p").text().trim();

      let publishTime = detail$("span.date").text().trim();
      publishTime = moment(publishTime, "YYYY年MM月DD日 HH:mm").format("YYYY-MM-DD HH:mm:ss");

      const imageUrl = detail$("div.img_wrapper img").attr("src");
      const fullImageUrl = getAbsoluteImageUrl(item.link, imageUrl);
      const imagePath = await downloadImage(fullImageUrl, item.title);

      const insertSql = `
        INSERT INTO news_items(title, content, publish_time, image_url)
        VALUES (?, ?, ?, ?)
      `;

      await new Promise((resolve, reject) => {
        connection.query(
          insertSql,
          [item.title, content, publishTime, imagePath],
          (err, results) => {
            if (err) {
              console.error("插入失败:", err);
            } else {
              console.log("插入成功:", results.insertId);
            }
            resolve();
          }
        );
      });
    }

  } catch (error) {
    console.error("爬取过程中出错:", error);
  } finally {
    if (connection && !connection._closing) {
      await new Promise((resolve) => {
        connection.end((err) => {
          if (err) console.error("关闭连接时出错:", err);
          else console.log("数据库连接已关闭。");
          resolve();
        });
      });
    }
  }
}

// 执行爬取
scrapeNews();
